import datetime
import json
import os
import re
import sys
from copy import deepcopy

import pytz

from feature_set.sms.un.sms_en_cate0_v1.SmsEnCate0V1ClfModel import SmsEnCate0V1ClfModel
from feature_set.sms.un.sms_en_loan0_v1.SmsEnLoan0V1ClfModel import SmsEnLoan0V1ClfModel
from feature_set.sms.un.sms_en_loan0_v1.SmsEnLoan0V1NerModel import SmsEnLoan0V1NerModel


def init_org_map(raw_org_map):
    org_reg_dict = {}
    for k, v in raw_org_map.items():
        if k == "Don":
            reg = r"[{\[\(<] Don [}\]\)>]"
        elif k == "facil":
            reg = r"[{\[\(<] facil [}\]\)>]"
        elif k == "Facil":
            reg = r"[{\[\(<] Facil [}\]\)>]"
        elif k == "Tiempo":
            reg = r"[{\[\(<] Tiempo [}\]\)>]"
        else:
            reg = re.compile(rf"\b{k}\b")
        org_reg_dict[reg] = v
    return org_reg_dict


class SmsEnLoan0V1:
    
    def __init__(self):
        super().__init__()
        self.root_dir = self.get_root_dir(os.path.abspath("."))
        self.conf_dir = os.path.join(self.root_dir, "feature_conf")

        self.cate_list_en = ['media', 'loan', 'bank', 'other', 'catering', 'ips', 'kyc', 'bet', 'edu', 'solar', 'energy',
                              'gov', 'medical', 'social', 'charity', 'retail', 'agric', 'prop', 'express', 'pay', 'finance', 
                              'internet', 'job', 'device', 'ins', 'travel', 'ss']

        self.featuremap = {
            "cnt": self.cal_cnt_feature,
            "amt": self.cal_amt_feature,
            "days": self.cal_days_feature,
            "orgcnt": self.cal_orgcnt_feature,
            "maxmininorg": self.cal_maxmininorg_feature,
        }

        self.tokenizer = Tokenizer()

        self.day_section = [(15, "d15"), (30, "m1"), (90, "m3"), (sys.maxsize, "all")]

        self.country_info = None
        self.body_name = None
        self.type_name = None
        self.phone_name = None
        self.time_name = None

    def gen_features(self, request_data):
        msg_list = self.load_request(request_data)
        rel = self.get_features(msg_list)
        rel = {k: (-999 if v == -1 else v) for k, v in rel.items()}
        return rel

    def load_conf(self):
        self.org_map = init_org_map(self.country_info["org_map"])
        self.rule_keywords_dict = self.country_info["loan_rule"]
        self.loan_clf_model = SmsEnLoan0V1ClfModel()
        self.loan_ner_model = SmsEnLoan0V1NerModel()
        self.cate_clf_model = SmsEnCate0V1ClfModel()

    def load_request(self, request_data):
        apply_time = self.trans_str_to_time(request_data.apply_time)

        try:
            sms_list = []
            sms_data = request_data.data_sources["sms_data"]
            sms_list = json.loads(sms_data) if type(sms_data) == str else sms_data
        except:
            pass

        if len(sms_list) == 0 or type(sms_list) != list:
            assert (
                1 == 0
            ), "传入的sms_json无法解析或者解析后异常(不是list，或者为空list)"

        name_trans_hash = {
            "sender": self.phone_name,
            "content": self.body_name,
            "type": self.type_name,
            "sms_time": self.time_name,
        }

        trans_sms_list = []
        for sms in sms_list:
            if (
                self.phone_name not in sms
                or self.body_name not in sms
                or self.type_name not in sms
                or self.time_name not in sms
                or sms[self.time_name] == ""
            ):
                continue
            trans_sms = {}
            trans_sms["sender"] = sms[name_trans_hash["sender"]]
            trans_sms["sms_time"] = datetime.datetime.utcfromtimestamp(
                int(sms[name_trans_hash["sms_time"]]) // 1000
            ) + datetime.timedelta(hours=self.country_info["time_zone"])

            trans_sms["type"] = int(sms[name_trans_hash["type"]])
            trans_sms["content"] = sms[name_trans_hash["content"]]
            trans_sms["diff_days"] = (
                apply_time.date() - trans_sms["sms_time"].date()
            ).days
            trans_sms_list.append(deepcopy(trans_sms))
        sorted_msg_list = sorted(
            trans_sms_list, key=lambda x: x["sms_time"], reverse=True
        )[:3000]
        clean_sorted_sms_list = [
            trans_sms
            for trans_sms in sorted_msg_list
            if trans_sms["sms_time"] < self.trans_str_to_time(request_data.apply_time)
            and trans_sms["type"] == 1
        ]
        return clean_sorted_sms_list

    def get_features(self, msg_list: list) -> dict:
        cate_list = self.msg_classify(msg_list)
        tag_list = [{} for _ in range(len(msg_list))]

        for idx in range(len(msg_list)):
            tag_list[idx]["diff_days"] = msg_list[idx]["diff_days"]
            tag_list[idx]["sender"] = msg_list[idx]["sender"]
            tag_list[idx]["sms_time"] = msg_list[idx]["sms_time"]

        self.mark_compete(msg_list, cate_list, tag_list)

        self.msg_tokenizer(msg_list, tag_list)

        self.mark_class(tag_list)

        self.mark_org(tag_list)

        self.mark_entity(tag_list)

        self.amt_cache_dict, self.cnt_cache_dict, self.days_cache_dict = (
            self.cache_data(tag_list)
        )

        res_map = {}
        for prefix, fun in self.featuremap.items():
            res_map.update(
                {
                    prefix + "_" + feature_name: feature_value
                    for feature_name, feature_value in fun().items()
                }
            )
        return res_map

    def init_cnt_features(self):
        feature_dict = {}
        class_list = ["ad", "col", "ovd", "loan", "ntf", "otr", "repy", "ver"]
        for ds in self.day_section:
            for label in class_list + ["all"]:
                feature_dict[self.generate_feature_name([label, "cnt", ds[1]])] = 0
                feature_dict[self.generate_feature_name([label, "phonecnt", ds[1]])] = (
                    -1
                )
                if label != "all":
                    feature_dict[
                        self.generate_feature_name([label, "cnt_rto", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "phonecnt_rto", ds[1]])
                    ] = -1
                if label in ["col", "ovd", "loan", "repy"]:
                    feature_dict[
                        self.generate_feature_name([label, "wtcnt", ds[1]])
                    ] = 0.0
            feature_dict[self.generate_feature_name(["catecnt", ds[1]])] = 0
            feature_dict[self.generate_feature_name(["topcate_cnt", ds[1]])] = 0
            feature_dict[self.generate_feature_name(["topcate_cnt_rto", ds[1]])] = -1
            feature_dict[self.generate_feature_name(["ad_ver_diff_cnt", ds[1]])] = -9999
        return feature_dict

    def init_amt_features(self):
        feature_dict = {}
        class_list = ["ad", "col", "ovd", "loan", "ntf", "otr", "repy", "ver"]
        for ds in self.day_section:
            for label in class_list + ["all"]:
                if label in ["col", "ovd", "loan", "repy"]:
                    feature_dict[
                        self.generate_feature_name([label, "amt_sum", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "amt_avg", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "amt_max", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "amt_min", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "amt_wtsum", ds[1]])
                    ] = -1
        return feature_dict

    ###激活maxmininorg特征
    def init_maxmininorg_features(self):
        feature_dict = {}
        class_list = ["ad", "col", "ovd", "loan", "ntf", "otr", "repy", "ver"]
        for ds in self.day_section:
            for label in class_list + ["all"]:
                feature_dict[
                    self.generate_feature_name([label, "maxinorg_cnt", ds[1]])
                ] = -1
                feature_dict[
                    self.generate_feature_name([label, "mininorg_cnt", ds[1]])
                ] = -1
                feature_dict[
                    self.generate_feature_name([label, "maxinorg_phonecnt", ds[1]])
                ] = -1
                feature_dict[
                    self.generate_feature_name([label, "mininorg_phonecnt", ds[1]])
                ] = -1
                if label in ["col", "ovd", "loan", "repy"]:
                    feature_dict[
                        self.generate_feature_name([label, "maxinorg_amt_sum", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "maxinorg_amt_avg", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "maxinorg_amt_min", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "mininorg_amt_sum", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "mininorg_amt_avg", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "mininorg_amt_max", ds[1]])
                    ] = -1
                if label in ["col", "ovd"]:
                    feature_dict[
                        self.generate_feature_name([label, "maxinorg_days_sum", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "maxinorg_days_avg", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "maxinorg_days_min", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "mininorg_days_sum", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "mininorg_days_avg", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "mininorg_days_max", ds[1]])
                    ] = -1
        return feature_dict

    ###激活天數類特征
    def init_days_features(self):
        feature_dict = {}
        class_list = ["ad", "col", "ovd", "loan", "ntf", "otr", "repy", "ver"]
        for ds in self.day_section:
            for label in class_list + ["all"]:
                if label in ["col", "ovd"]:
                    feature_dict[
                        self.generate_feature_name([label, "days_sum", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "days_avg", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "days_max", ds[1]])
                    ] = -1
                    feature_dict[
                        self.generate_feature_name([label, "days_min", ds[1]])
                    ] = -1
        return feature_dict

    ###激活機構類特征
    def init_orgcnt_features(self):
        feature_dict = {}
        class_list = ["ad", "col", "ovd", "loan", "ntf", "otr", "repy", "ver"]
        for ds in self.day_section:
            for label in class_list + ["all"]:
                if label != "all":
                    feature_dict[
                        self.generate_feature_name([label, "orgcnt_rto", ds[1]])
                    ] = -1
                feature_dict[self.generate_feature_name([label, "orgcnt", ds[1]])] = -1
            feature_dict[self.generate_feature_name(["ad_ver_diff_orgcnt", ds[1]])] = (
                -9999
            )
        return feature_dict

    ###下面是計算各子模塊特征的函數
    def cal_cnt_feature(self):
        cnt_cache_dict = deepcopy(self.cnt_cache_dict)
        loan_features = self.init_cnt_features()
        for ds in cnt_cache_dict:
            loan_features[self.generate_feature_name(["catecnt", ds])] = (
                len(cnt_cache_dict[ds]) - 1
            )
            for msg_class in cnt_cache_dict[ds]:
                phone_list = []
                msg_class_cnt = 0
                msg_class_wtcnt = 0
                for org in cnt_cache_dict[ds][msg_class]:
                    msg_class_cnt += len(cnt_cache_dict[ds][msg_class][org])
                    for i in cnt_cache_dict[ds][msg_class][org]:
                        phone_list.append(i[2])
                    if msg_class in ["col", "ovd", "loan", "repy"]:
                        for i in cnt_cache_dict[ds][msg_class][org]:
                            msg_class_wtcnt += i[0] * (0.97 ** i[1])
                phone_set = set(phone_list)
                loan_features[self.generate_feature_name([msg_class, "cnt", ds])] = (
                    msg_class_cnt
                )
                if msg_class in ["col", "ovd", "loan", "repy"]:
                    loan_features[
                        self.generate_feature_name([msg_class, "wtcnt", ds])
                    ] = msg_class_wtcnt
                loan_features[
                    self.generate_feature_name([msg_class, "phonecnt", ds])
                ] = len(phone_set)
            for msg_class in cnt_cache_dict[ds]:
                if msg_class != "all":
                    loan_features[
                        self.generate_feature_name([msg_class, "cnt_rto", ds])
                    ] = (
                        loan_features[
                            self.generate_feature_name([msg_class, "cnt", ds])
                        ]
                        / loan_features[self.generate_feature_name(["cnt", ds])]
                        if loan_features[self.generate_feature_name(["cnt", ds])] > 3
                        else -1
                    )

                    loan_features[
                        self.generate_feature_name([msg_class, "phonecnt_rto", ds])
                    ] = (
                        loan_features[
                            self.generate_feature_name([msg_class, "phonecnt", ds])
                        ]
                        / loan_features[self.generate_feature_name(["phonecnt", ds])]
                        if loan_features[self.generate_feature_name(["phonecnt", ds])]
                        != 0
                        else -1
                    )

                    if (
                        loan_features[self.generate_feature_name(["topcate_cnt", ds])]
                        < loan_features[
                            self.generate_feature_name([msg_class, "cnt", ds])
                        ]
                    ):
                        loan_features[
                            self.generate_feature_name(["topcate_cnt", ds])
                        ] = loan_features[
                            self.generate_feature_name([msg_class, "cnt", ds])
                        ]
                        loan_features[
                            self.generate_feature_name(["topcate_cnt_rto", ds])
                        ] = (
                            loan_features[
                                self.generate_feature_name(["topcate_cnt", ds])
                            ]
                            / loan_features[self.generate_feature_name(["cnt", ds])]
                            if loan_features[self.generate_feature_name(["cnt", ds])]
                            > 3
                            else -1
                        )
            loan_features[self.generate_feature_name(["ad_ver_diff_cnt", ds])] = (
                loan_features[self.generate_feature_name(["ad", "cnt", ds])]
                - loan_features[self.generate_feature_name(["ver", "cnt", ds])]
            )
        return loan_features

    def cal_amt_feature(self):
        amt_cache_dict = deepcopy(self.amt_cache_dict)
        loan_features = self.init_amt_features()
        for ds in amt_cache_dict:
            for msg_class in amt_cache_dict[ds]:
                if msg_class in ["col", "ovd", "loan", "repy"]:
                    msg_class_amt_sum = 0
                    msg_class_amt_cnt = 0
                    msg_class_amt_wtsum = 0
                    msg_class_amt_min = float("inf")
                    msg_class_amt_max = -float("inf")
                    flag = False
                    for org in amt_cache_dict[ds][msg_class]:
                        if len(amt_cache_dict[ds][msg_class][org]) != 0:
                            flag = True
                            sum_org_amt = sum(
                                x[0] for x in amt_cache_dict[ds][msg_class][org]
                            )
                            wtsum_org_amt = sum(
                                x[0] * (0.97 ** x[1])
                                for x in amt_cache_dict[ds][msg_class][org]
                            )
                            cnt_org_amt = len(amt_cache_dict[ds][msg_class][org])
                            max_org_amt = max(
                                amt_cache_dict[ds][msg_class][org], key=lambda x: x[0]
                            )[0]
                            min_org_amt = min(
                                amt_cache_dict[ds][msg_class][org], key=lambda x: x[0]
                            )[0]
                            msg_class_amt_sum += sum_org_amt
                            msg_class_amt_wtsum += wtsum_org_amt
                            msg_class_amt_cnt += cnt_org_amt
                            msg_class_amt_min = min(msg_class_amt_min, min_org_amt)
                            msg_class_amt_max = max(msg_class_amt_max, max_org_amt)
                    msg_class_amt_avg = (
                        msg_class_amt_sum / msg_class_amt_cnt
                        if msg_class_amt_cnt != 0
                        else 0
                    )
                    loan_features[
                        self.generate_feature_name([msg_class, "amt_sum", ds])
                    ] = (msg_class_amt_sum if flag else -1)
                    loan_features[
                        self.generate_feature_name([msg_class, "amt_wtsum", ds])
                    ] = (msg_class_amt_wtsum if flag else -1)
                    loan_features[
                        self.generate_feature_name([msg_class, "amt_avg", ds])
                    ] = (msg_class_amt_avg if flag else -1)
                    loan_features[
                        self.generate_feature_name([msg_class, "amt_max", ds])
                    ] = (msg_class_amt_max if flag else -1)
                    loan_features[
                        self.generate_feature_name([msg_class, "amt_min", ds])
                    ] = (msg_class_amt_min if flag else -1)
        return loan_features

    def cal_days_feature(self):
        days_cache_dict = deepcopy(self.days_cache_dict)
        loan_features = self.init_days_features()
        for ds in days_cache_dict:
            for msg_class in days_cache_dict[ds]:
                if msg_class in ["col", "ovd"]:
                    msg_class_days_sum = 0
                    msg_class_days_cnt = 0
                    msg_class_days_min = float("inf")
                    msg_class_days_max = -float("inf")
                    flag = False
                    for org in days_cache_dict[ds][msg_class]:
                        if len(days_cache_dict[ds][msg_class][org]) == 0:
                            continue
                        elif len(days_cache_dict[ds][msg_class][org]) != 0:
                            flag = True
                        sum_org_days = sum(
                            x[0] for x in days_cache_dict[ds][msg_class][org]
                        )
                        cnt_org_days = len(days_cache_dict[ds][msg_class][org])
                        max_org_days = max(
                            days_cache_dict[ds][msg_class][org], key=lambda x: x[0]
                        )[0]
                        min_org_days = min(
                            days_cache_dict[ds][msg_class][org], key=lambda x: x[0]
                        )[0]
                        msg_class_days_sum += sum_org_days
                        msg_class_days_cnt += cnt_org_days
                        msg_class_days_min = min(msg_class_days_min, max_org_days)
                        msg_class_days_max = max(msg_class_days_max, min_org_days)
                    msg_class_days_avg = (
                        msg_class_days_sum / msg_class_days_cnt
                        if msg_class_days_cnt != 0
                        else 0
                    )
                    loan_features[
                        self.generate_feature_name([msg_class, "days_sum", ds])
                    ] = msg_class_days_sum
                    loan_features[
                        self.generate_feature_name([msg_class, "days_avg", ds])
                    ] = msg_class_days_avg
                    loan_features[
                        self.generate_feature_name([msg_class, "days_max", ds])
                    ] = (msg_class_days_max if flag else -1)
                    loan_features[
                        self.generate_feature_name([msg_class, "days_min", ds])
                    ] = (msg_class_days_min if flag else -1)
        return loan_features

    def cal_orgcnt_feature(self):
        cnt_cache_dict = deepcopy(self.cnt_cache_dict)
        loan_features = self.init_orgcnt_features()
        for ds in cnt_cache_dict:
            for msg_class in cnt_cache_dict[ds]:
                msg_class_orgcnt = (
                    len(cnt_cache_dict[ds][msg_class])
                    if "none" not in cnt_cache_dict[ds][msg_class]
                    else len(cnt_cache_dict[ds][msg_class]) - 1
                )
                loan_features[self.generate_feature_name([msg_class, "orgcnt", ds])] = (
                    msg_class_orgcnt
                )
            for msg_class in cnt_cache_dict[ds]:
                if msg_class != "all":
                    loan_features[
                        self.generate_feature_name([msg_class, "orgcnt_rto", ds])
                    ] = (
                        loan_features[
                            self.generate_feature_name([msg_class, "orgcnt", ds])
                        ]
                        / loan_features[self.generate_feature_name(["orgcnt", ds])]
                        if loan_features[self.generate_feature_name(["orgcnt", ds])]
                        != 0
                        else -1
                    )
            loan_features[self.generate_feature_name(["ad_ver_diff_orgcnt", ds])] = (
                loan_features[self.generate_feature_name(["ad", "orgcnt", ds])]
                - loan_features[self.generate_feature_name(["ver", "orgcnt", ds])]
            )
        return loan_features

    def cal_maxmininorg_feature(self):
        cnt_cache_dict = deepcopy(self.cnt_cache_dict)
        amt_cache_dict = deepcopy(self.amt_cache_dict)
        days_cache_dict = deepcopy(self.days_cache_dict)
        loan_features = self.init_maxmininorg_features()
        for ds in cnt_cache_dict:
            for msg_class in cnt_cache_dict[ds]:
                maxinorg_class_cnt = 0
                mininorg_class_cnt = float("inf")
                maxinorg_phone_cnt = 0
                mininorg_phone_cnt = float("inf")
                flag = False
                for org in cnt_cache_dict[ds][msg_class]:
                    if org == "none":
                        continue
                    elif org != "none" and len(cnt_cache_dict[ds][msg_class][org]) != 0:
                        flag = True
                        org_msg_cnt = len(cnt_cache_dict[ds][msg_class][org])
                        maxinorg_class_cnt = max(maxinorg_class_cnt, org_msg_cnt)
                        mininorg_class_cnt = min(mininorg_class_cnt, org_msg_cnt)
                        org_phone_list = []
                        for i in cnt_cache_dict[ds][msg_class][org]:
                            org_phone_list.append(i[2])
                        org_phone_set = set(org_phone_list)
                        org_phone_cnt = len(org_phone_set)
                        maxinorg_phone_cnt = max(maxinorg_phone_cnt, org_phone_cnt)
                        mininorg_phone_cnt = max(mininorg_phone_cnt, org_phone_cnt)
                loan_features[
                    self.generate_feature_name([msg_class, "maxinorg", "cnt", ds])
                ] = (maxinorg_class_cnt if flag else -1)
                loan_features[
                    self.generate_feature_name([msg_class, "mininorg", "cnt", ds])
                ] = (mininorg_class_cnt if flag else -1)
                loan_features[
                    self.generate_feature_name([msg_class, "maxinorg", "phonecnt", ds])
                ] = (maxinorg_class_cnt if flag else -1)
                loan_features[
                    self.generate_feature_name([msg_class, "mininorg", "phonecnt", ds])
                ] = (mininorg_class_cnt if flag else -1)

            for msg_class in amt_cache_dict[ds]:
                if msg_class not in ["col", "ovd", "loan", "repy"]:
                    continue
                maxinorg_class_amt_sum = -float("inf")
                maxinorg_class_amt_avg = -float("inf")
                maxinorg_class_amt_min = -float("inf")
                mininorg_class_amt_sum = float("inf")
                mininorg_class_amt_avg = float("inf")
                mininorg_class_amt_max = float("inf")
                flag = False
                for org in amt_cache_dict[ds][msg_class]:
                    if org == "none":
                        continue
                    elif org != "none" and len(amt_cache_dict[ds][msg_class][org]) != 0:
                        flag = True
                        org_msg_cnt = len(amt_cache_dict[ds][msg_class][org])
                        org_amt_sum = sum(
                            [x[0] for x in amt_cache_dict[ds][msg_class][org]]
                        )
                        org_amt_avg = org_amt_sum / org_msg_cnt
                        org_amt_max = max(
                            amt_cache_dict[ds][msg_class][org], key=lambda x: x[0]
                        )[0]
                        org_amt_min = min(
                            amt_cache_dict[ds][msg_class][org], key=lambda x: x[0]
                        )[0]
                        maxinorg_class_amt_sum = max(
                            maxinorg_class_amt_sum, org_amt_sum
                        )
                        maxinorg_class_amt_avg = max(
                            maxinorg_class_amt_avg, org_amt_avg
                        )
                        maxinorg_class_amt_min = max(
                            maxinorg_class_amt_min, org_amt_min
                        )
                        mininorg_class_amt_sum = min(
                            mininorg_class_amt_sum, org_amt_sum
                        )
                        mininorg_class_amt_avg = min(
                            mininorg_class_amt_avg, org_amt_avg
                        )
                        mininorg_class_amt_max = min(
                            mininorg_class_amt_max, org_amt_max
                        )
                loan_features[
                    self.generate_feature_name([msg_class, "maxinorg", "amt_sum", ds])
                ] = (maxinorg_class_amt_sum if flag else -1)
                loan_features[
                    self.generate_feature_name([msg_class, "maxinorg", "amt_avg", ds])
                ] = (maxinorg_class_amt_avg if flag else -1)
                loan_features[
                    self.generate_feature_name([msg_class, "maxinorg", "amt_min", ds])
                ] = (maxinorg_class_amt_min if flag else -1)
                loan_features[
                    self.generate_feature_name([msg_class, "mininorg", "amt_sum", ds])
                ] = (mininorg_class_amt_sum if flag else -1)
                loan_features[
                    self.generate_feature_name([msg_class, "mininorg", "amt_avg", ds])
                ] = (mininorg_class_amt_avg if flag else -1)
                loan_features[
                    self.generate_feature_name([msg_class, "mininorg", "amt_max", ds])
                ] = (mininorg_class_amt_max if flag else -1)

            for msg_class in days_cache_dict[ds]:
                if msg_class not in ["col", "ovd"]:
                    continue
                maxinorg_class_days_sum = -float("inf")
                maxinorg_class_days_avg = -float("inf")
                maxinorg_class_days_min = -float("inf")
                mininorg_class_days_sum = float("inf")
                mininorg_class_days_avg = float("inf")
                mininorg_class_days_max = float("inf")
                flag = False
                for org in days_cache_dict[ds][msg_class]:
                    if org == "none":
                        continue
                    elif (
                        org != "none" and len(days_cache_dict[ds][msg_class][org]) != 0
                    ):
                        flag = True
                        org_msg_cnt = len(days_cache_dict[ds][msg_class][org])
                        org_days_sum = sum(
                            [x[0] for x in days_cache_dict[ds][msg_class][org]]
                        )
                        org_days_avg = org_days_sum / org_msg_cnt
                        org_days_max = max(
                            days_cache_dict[ds][msg_class][org], key=lambda x: x[0]
                        )[0]
                        org_days_min = min(
                            days_cache_dict[ds][msg_class][org], key=lambda x: x[0]
                        )[0]
                        maxinorg_class_days_sum = max(
                            maxinorg_class_days_sum, org_days_sum
                        )
                        maxinorg_class_days_avg = max(
                            maxinorg_class_days_avg, org_days_avg
                        )
                        maxinorg_class_days_min = max(
                            maxinorg_class_days_min, org_days_min
                        )
                        mininorg_class_days_sum = min(
                            mininorg_class_days_sum, org_days_sum
                        )
                        mininorg_class_days_avg = min(
                            mininorg_class_days_avg, org_days_avg
                        )
                        mininorg_class_days_max = min(
                            mininorg_class_days_max, org_days_max
                        )
                loan_features[
                    self.generate_feature_name([msg_class, "maxinorg", "days_sum", ds])
                ] = (maxinorg_class_days_sum if flag else -1)
                loan_features[
                    self.generate_feature_name([msg_class, "maxinorg", "days_avg", ds])
                ] = (maxinorg_class_days_avg if flag else -1)
                loan_features[
                    self.generate_feature_name([msg_class, "maxinorg", "days_min", ds])
                ] = (maxinorg_class_days_min if flag else -1)
                loan_features[
                    self.generate_feature_name([msg_class, "mininorg", "days_sum", ds])
                ] = (mininorg_class_days_sum if flag else -1)
                loan_features[
                    self.generate_feature_name([msg_class, "mininorg", "days_avg", ds])
                ] = (mininorg_class_days_avg if flag else -1)
                loan_features[
                    self.generate_feature_name([msg_class, "mininorg", "days_max", ds])
                ] = (mininorg_class_days_max if flag else -1)
        return loan_features


    def normalize_entities(self, entities_dict):
        normalize_entities = {}
        if "AMOUNT" in entities_dict:
            normalize_entities["AMOUNT"] = self.normalize_amount(
                entities_dict["AMOUNT"]
            )
        if "TERM" in entities_dict:
            normalize_entities["DAYS"] = self.normalize_days(entities_dict["TERM"])
        if "DEDUCTION" in entities_dict:
            normalize_entities["AMOUNT"] = self.normalize_amount(
                entities_dict["DEDUCTION"]
            )
        if "DATE" in entities_dict:
            normalize_entities["DATE"] = self.normalize_date(entities_dict["DATE"])
        return normalize_entities

    def normalize_amount(self, content):
        clean_list = []
        normalized_amount = 0
        for c in content.replace(',', '.').replace('.0000', '').replace('s0l',''):
            if c.isdigit() or c == '.':
                clean_list.append(c)
        clean_str = ''.join(clean_list)
        if len(clean_str) == 0:
            return normalized_amount
        split_arr = clean_str.rsplit('.', 1)
        split_arr[0] = split_arr[0].replace('.', '')
        if len(split_arr) == 1:
            if float(split_arr[0]) < 1000000:
                normalized_amount = float(split_arr[0])
            return normalized_amount
        elif len(split_arr) == 2:
            if len(split_arr[1]) >= 3:
                if float("%s%s" % (split_arr[0], split_arr[1])) < 1000000:
                    normalized_amount = float("%s%s" % (split_arr[0], split_arr[1]))
                return normalized_amount
            elif split_arr[0] == '' and split_arr[1] == '':
                    return normalized_amount
            else:
                if float("%s.%s" % (split_arr[0], split_arr[1])) < 1000000:
                    normalized_amount = float("%s.%s" % (split_arr[0], split_arr[1]))
                return normalized_amount
        return normalized_amount

    def normalize_days(self, content):
        content = content.lower()
        if content in ['today']:
            return 0
        elif content in ['tomorrow', 'yesterday']:
            return 1
        else:
            if content.isdigit():
                if float(content) < 100:
                    return float(content)
            else:
                content = re.sub('[^0-9]','',content)
                if content.isdigit():
                    if float(content) < 100:
                        return float(content)
        return 0

    def normalize_date(self, content):
        content = content.replace(' - ','-').lower()
        date_formats = [
            "%d %b %Y",
            "%Y-%m-%d",
            '%d-%m-%Y'
        ]
        pattern = r"[^a-z0-9-\s]"
        content = re.sub(pattern, '', content)
        content = re.sub('\s+', ' ', content)
        for date_format in date_formats:
            try:
                date_object = datetime.strptime(content, date_format)
                standard_content = date_object.strftime("%Y-%m-%d")
                return standard_content
            except:
                pass
        return None

    def mark_compete(self, msg_list, cate_list, tag_list):
        for idx in range(len(cate_list)):
            is_compete = True if cate_list[idx] == "loan" else False
            if is_compete:
                for acc_kw in self.rule_keywords_dict["acc_rule"]:
                    if acc_kw in msg_list[idx]["content"]:
                        is_compete = False
                        break
            else:
                for rec_kw in self.rule_keywords_dict["recall_rule"]:
                    if rec_kw in msg_list[idx]["content"]:
                        is_compete = True
                        break
            tag_list[idx]["is_compete"] = is_compete

    def mark_class(self, tag_list):
        input_text_list = [x["process_msg"] for x in tag_list if x["is_compete"]]
        input_text_list_uniq = list(set(input_text_list))
        if len(input_text_list_uniq) > 0:
            predict_class_list = self.loan_clf_model.predict(input_text_list_uniq)
        else:
            predict_class_list = []
        predict_dict = dict(zip(input_text_list_uniq, predict_class_list))
        for tag in tag_list:
            if tag["is_compete"]:
                if predict_dict[tag["process_msg"]] != 'apf':
                    tag["class"] = predict_dict[tag["process_msg"]]
                else:
                    tag["class"] = 'ntf'
            else:
                tag["class"] = None


    def mark_entity(self, tag_list):
        input_text_list = [
            x["process_msg"]
            for x in tag_list
            if x["is_compete"] and x["class"] in ["col", "ovd", "repy", "loan"]
        ]
        input_text_list_uniq = list(set(input_text_list))
        predict_list = self.loan_ner_model.predict(input_text_list_uniq)
        predict_dict = dict(zip(input_text_list_uniq, predict_list))
        for tag in tag_list:
            tag["entity"] = (
                predict_dict[tag["process_msg"]]
                if tag["is_compete"] and tag["class"] in ["col", "ovd", "repy", "loan"]
                else {}
            )
            tag["entity"] = self.normalize_entities(tag["entity"])

    def mark_org(self, tag_list):
        for tag in tag_list:
            tag["org"] = "none"
            if tag["is_compete"]:
                for reg in self.org_map:
                    if re.search(reg, tag["process_msg"]):
                        tag["org"] = self.org_map[reg]
                        break

    def cache_data(self, tag_list):
        amt_cache_dict = {}
        cnt_cache_dict = {}
        days_cache_dict = {}
        for tag in tag_list:
            if not tag["is_compete"]:
                continue
            for win in self.day_section:
                if tag["diff_days"] > win[0]:
                    continue
                ds = win[1]
                if ds not in amt_cache_dict:
                    amt_cache_dict[ds] = {}
                if ds not in cnt_cache_dict:
                    cnt_cache_dict[ds] = {}
                if ds not in days_cache_dict:
                    days_cache_dict[ds] = {}
                msg_class_list = [tag["class"], "all"]
                for msg_class in msg_class_list:
                    msg_entities = tag["entity"]
                    # amount dict
                    if msg_class != "all":
                        if msg_class not in amt_cache_dict[ds]:
                            amt_cache_dict[ds][msg_class] = {}
                        msg_org = tag["org"]
                        if msg_org not in amt_cache_dict[ds][msg_class]:
                            amt_cache_dict[ds][msg_class][msg_org] = []
                        if "AMOUNT" in msg_entities:
                            amt_cache_dict[ds][msg_class][msg_org].append(
                                (msg_entities["AMOUNT"], tag["diff_days"])
                            )
                    if msg_class not in cnt_cache_dict[ds]:
                        cnt_cache_dict[ds][msg_class] = {}
                    msg_org = tag["org"]
                    if msg_org not in cnt_cache_dict[ds][msg_class]:
                        cnt_cache_dict[ds][msg_class][msg_org] = []
                    cnt_cache_dict[ds][msg_class][msg_org].append(
                        (1, tag["diff_days"], tag["sender"])
                    )

                    if msg_class != "all":

                        if msg_class not in days_cache_dict[ds]:
                            days_cache_dict[ds][msg_class] = {}
                        msg_org = tag["org"]
                        if msg_org not in days_cache_dict[ds][msg_class]:
                            days_cache_dict[ds][msg_class][msg_org] = []
                        if "DAYS" in msg_entities:
                            days_cache_dict[ds][msg_class][msg_org].append(
                                (msg_entities["DAYS"], tag["diff_days"])
                            )
                        elif "DAYS" not in msg_entities and "DATE" in msg_entities:
                            if msg_entities["DATE"]:
                                date = datetime.datetime.strptime(
                                    msg_entities["DATE"], "%Y-%m-%d"
                                )
                                diffdays = (
                                    date.replace(tzinfo=None)
                                    - tag["sms_time"].replace(tzinfo=None)
                                ).days
                                days_cache_dict[ds][msg_class][msg_org].append(
                                    (diffdays, tag["diff_days"])
                                )
        return amt_cache_dict, cnt_cache_dict, days_cache_dict

    def msg_tokenizer(self, msg_list, tag_list):
        for idx in range(len(msg_list)):
            tokens, process_msg = None, None
            if tag_list[idx]["is_compete"]:
                tokens = self.tokenizer.tokenize(msg_list[idx]["content"])
                process_msg = " ".join(tokens)
            tag_list[idx]["process_msg"] = process_msg
            tag_list[idx]["tokens"] = tokens

    def msg_classify(self, msg_list):
        predict_cate_list = ['none'] * len(msg_list)
        cate_clf_conf = self.country_info['sms_cate_clf_conf']
        input_text_list = []
        for idx, msg in enumerate(msg_list):
            if msg['sender'] in cate_clf_conf:
                predict_cate_list[idx] = cate_clf_conf[msg['sender']]
            else:
                input_text = msg['content']
                res = re.findall('[A-Za-z]', msg['sender'], re.I)
                if res:
                    input_text = "%s %s" % (msg['sender'], input_text)
                input_text_list.append(input_text)
        if len(input_text_list) > 0:
            input_text_list_uniq = list(set(input_text_list))
            tags, prob = self.cate_clf_model.predict(input_text_list_uniq)
            predict_dict = dict(zip(input_text_list_uniq, tags))
            predict_cate_list_by_model = [self.cate_list_en[predict_dict[x]] for x in input_text_list]
            idx = 0
            for i in range(len(predict_cate_list)):
                if predict_cate_list[i] == 'none':
                    predict_cate_list[i] = predict_cate_list_by_model[idx]
                    idx += 1
        assert len(predict_cate_list) == len(msg_list)
        return predict_cate_list

    def generate_feature_name(self, tag_list):
        clean_tag_list = []
        for tag in tag_list:
            tag = tag.replace("all", "")
            if tag != "":
                clean_tag_list.append(tag)
        return "_".join(clean_tag_list)

    def trans_str_to_time(self, str_time):
        return datetime.datetime.strptime(str_time, "%Y-%m-%d %H:%M:%S")

    def get_root_dir(self,path):
        path_list=path.split(os.path.sep)
        index=path_list.index("featurelib")
        return os.path.sep.join(path_list[:index+1])

class Tokenizer:
    def __init__(self):
        self.pat = r"[^\-\_\,\.\<\>\[\]\¿\¡\!\"\#\$\%\&\\\'\(\)\*\+\/\:\;\<\=\>\?\@\\\\\^\`\{\|\}\~\，\。\？\！\：\、\《\》\ a-zA-Z\u00C0-\u00FF\d]+"
        self.capture_symbol = r"([\:\-\_\,\.\(\)\<\>\[\]\?\!\@])"
        self.non_capture_symbol = (
            r"[\¿\¡\"\#\$\%\&\\\'\*\+\/\;\=\\\\\^\`\{\|\}\~\，\。\？\！\：\、\《\》\ ]"
        )
        self.pattern = re.compile(
            "|".join([self.capture_symbol, self.non_capture_symbol])
        )

    def tokenize(self, sentence):
        sentence = re.sub(self.pat, " ", sentence)
        token_list = []
        tokens = re.split(self.pattern, sentence)
        for token in tokens:
            if token:
                token_list.append(token)
        return token_list
